Video Metadata MCP Server

{ "data": { "edges": [ { "animated": false, "className": "", "data": { "sourceHandle": { "dataType": "ChatInput", "id": "ChatInput-BLgCA", "name": "message", "output_types": [ "Message" ] }, "targetHandle": { "fieldName": "input_value", "id": "Agent-W6zzd", "inputTypes": [ "Message" ], "type": "str" } }, "id": "xy-edge__ChatInput-BLgCA{œdataTypeœ:œChatInputœ,œidœ:œChatInput-BLgCAœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-Agent-W6zzd{œfieldNameœ:œinput_valueœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", "selected": false, "source": "ChatInput-BLgCA", "sourceHandle": "{œdataTypeœ:œChatInputœ,œidœ:œChatInput-BLgCAœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}", "target": "Agent-W6zzd", "targetHandle": "{œfieldNameœ:œinput_valueœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { "dataType": "SearchOnVideo", "id": "SearchOnVideo-LY0Qv", "name": "component_as_tool", "output_types": [ "Tool" ] }, "targetHandle": { "fieldName": "tools", "id": "Agent-W6zzd", "inputTypes": [ "Tool" ], "type": "other" } }, "id": "xy-edge__SearchOnVideo-LY0Qv{œdataTypeœ:œSearchOnVideoœ,œidœ:œSearchOnVideo-LY0Qvœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}-Agent-W6zzd{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}", "selected": false, "source": "SearchOnVideo-LY0Qv", "sourceHandle": "{œdataTypeœ:œSearchOnVideoœ,œidœ:œSearchOnVideo-LY0Qvœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}", "target": "Agent-W6zzd", "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { "dataType": "Agent", "id": "Agent-W6zzd", "name": "response", "output_types": [ "Message" ] }, "targetHandle": { "fieldName": "input_value", "id": "ChatOutput-Cm5t9", "inputTypes": [ "Data", "DataFrame", "Message" ], "type": "other" } }, "id": "xy-edge__Agent-W6zzd{œdataTypeœ:œAgentœ,œidœ:œAgent-W6zzdœ,œnameœ:œresponseœ,œoutput_typesœ:[œMessageœ]}-ChatOutput-Cm5t9{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Cm5t9œ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", "selected": false, "source": "Agent-W6zzd", "sourceHandle": "{œdataTypeœ:œAgentœ,œidœ:œAgent-W6zzdœ,œnameœ:œresponseœ,œoutput_typesœ:[œMessageœ]}", "target": "ChatOutput-Cm5t9", "targetHandle": "{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Cm5t9œ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { "dataType": "CreateIndex", "id": "CreateSearchIndex-ynUMz", "name": "component_as_tool", "output_types": [ "Tool" ] }, "targetHandle": { "fieldName": "tools", "id": "Agent-W6zzd", "inputTypes": [ "Tool" ], "type": "other" } }, "id": "xy-edge__CreateSearchIndex-ynUMz{œdataTypeœ:œCreateIndexœ,œidœ:œCreateSearchIndex-ynUMzœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}-Agent-W6zzd{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}", "selected": false, "source": "CreateSearchIndex-ynUMz", "sourceHandle": "{œdataTypeœ:œCreateIndexœ,œidœ:œCreateSearchIndex-ynUMzœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}", "target": "Agent-W6zzd", "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { "dataType": "UploadVideo", "id": "UploadVideo-RxJXM", "name": "component_as_tool", "output_types": [ "Tool" ] }, "targetHandle": { "fieldName": "tools", "id": "Agent-W6zzd", "inputTypes": [ "Tool" ], "type": "other" } }, "id": "xy-edge__UploadVideo-RxJXM{œdataTypeœ:œUploadVideoœ,œidœ:œUploadVideo-RxJXMœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}-Agent-W6zzd{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}", "selected": false, "source": "UploadVideo-RxJXM", "sourceHandle": "{œdataTypeœ:œUploadVideoœ,œidœ:œUploadVideo-RxJXMœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}", "target": "Agent-W6zzd", "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-W6zzdœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" } ], "nodes": [ { "data": { "description": "Define the agent's instructions, then enter a task to complete using tools.", "display_name": "Agent", "id": "Agent-W6zzd", "node": { "base_classes": [ "Message" ], "beta": false, "conditional_paths": [], "custom_fields": {}, "description": "Define the agent's instructions, then enter a task to complete using tools.", "display_name": "Agent", "documentation": "", "edited": false, "field_order": [ "agent_llm", "max_tokens", "model_kwargs", "json_mode", "model_name", "openai_api_base", "api_key", "temperature", "seed", "max_retries", "timeout", "system_prompt", "tools", "input_value", "handle_parsing_errors", "verbose", "max_iterations", "agent_description", "memory", "sender", "sender_name", "n_messages", "session_id", "order", "template", "add_current_date_tool" ], "frozen": false, "icon": "bot", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Response", "hidden": false, "method": "message_response", "name": "response", "options": null, "required_inputs": null, "selected": "Message", "tool_mode": true, "types": [ "Message" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { "_type": "Component", "add_current_date_tool": { "_input_type": "BoolInput", "advanced": true, "display_name": "Current Date", "dynamic": false, "info": "If true, will add a tool to the agent that returns the current date.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "add_current_date_tool", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true }, "agent_description": { "_input_type": "MultilineInput", "advanced": true, "copy_field": false, "display_name": "Agent Description [Deprecated]", "dynamic": false, "info": "The description of the agent. This is only used when in Tool Mode. Defaults to 'A helpful assistant with access to the following tools:' and tools are added dynamically. This feature is deprecated and will be removed in future versions.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "multiline": true, "name": "agent_description", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "A helpful assistant with access to the following tools:" }, "agent_llm": { "_input_type": "DropdownInput", "advanced": false, "combobox": false, "dialog_inputs": {}, "display_name": "Model Provider", "dynamic": false, "info": "The provider of the language model that the agent will use to generate responses.", "input_types": [], "load_from_db": false, "name": "agent_llm", "options": [ "Amazon Bedrock", "Anthropic", "Azure OpenAI", "Google Generative AI", "Groq", "NVIDIA", "OpenAI", "SambaNova", "Custom" ], "options_metadata": [ { "icon": "Amazon" }, { "icon": "Anthropic" }, { "icon": "Azure" }, { "icon": "GoogleGenerativeAI" }, { "icon": "Groq" }, { "icon": "NVIDIA" }, { "icon": "OpenAI" }, { "icon": "SambaNova" }, { "icon": "brain" } ], "placeholder": "", "real_time_refresh": true, "required": false, "show": true, "title_case": false, "toggle": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "Google Generative AI" }, "api_key": { "_input_type": "SecretStrInput", "advanced": false, "display_name": "Google API Key", "dynamic": false, "info": "The Google API Key to use for the Google Generative AI.", "input_types": [], "load_from_db": false, "name": "api_key", "password": true, "placeholder": "", "real_time_refresh": true, "required": true, "show": true, "title_case": false, "type": "str", "value": "" }, "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "input_types": [], "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "from langchain_core.tools import StructuredTool\n\nfrom langflow.base.agents.agent import LCToolsAgentComponent\nfrom langflow.base.agents.events import ExceptionWithMessageError\nfrom langflow.base.models.model_input_constants import (\n ALL_PROVIDER_FIELDS,\n MODEL_DYNAMIC_UPDATE_FIELDS,\n MODEL_PROVIDERS_DICT,\n MODELS_METADATA,\n)\nfrom langflow.base.models.model_utils import get_model_name\nfrom langflow.components.helpers import CurrentDateComponent\nfrom langflow.components.helpers.memory import MemoryComponent\nfrom langflow.components.langchain_utilities.tool_calling import ToolCallingAgentComponent\nfrom langflow.custom.utils import update_component_build_config\nfrom langflow.io import BoolInput, DropdownInput, MultilineInput, Output\nfrom langflow.logging import logger\nfrom langflow.schema.dotdict import dotdict\nfrom langflow.schema.message import Message\n\n\ndef set_advanced_true(component_input):\n component_input.advanced = True\n return component_input\n\n\nclass AgentComponent(ToolCallingAgentComponent):\n display_name: str = \"Agent\"\n description: str = \"Define the agent's instructions, then enter a task to complete using tools.\"\n icon = \"bot\"\n beta = False\n name = \"Agent\"\n\n memory_inputs = [set_advanced_true(component_input) for component_input in MemoryComponent().inputs]\n\n inputs = [\n DropdownInput(\n name=\"agent_llm\",\n display_name=\"Model Provider\",\n info=\"The provider of the language model that the agent will use to generate responses.\",\n options=[*sorted(MODEL_PROVIDERS_DICT.keys()), \"Custom\"],\n value=\"OpenAI\",\n real_time_refresh=True,\n input_types=[],\n options_metadata=[MODELS_METADATA[key] for key in sorted(MODELS_METADATA.keys())] + [{\"icon\": \"brain\"}],\n ),\n *MODEL_PROVIDERS_DICT[\"OpenAI\"][\"inputs\"],\n MultilineInput(\n name=\"system_prompt\",\n display_name=\"Agent Instructions\",\n info=\"System Prompt: Initial instructions and context provided to guide the agent's behavior.\",\n value=\"You are a helpful assistant that can use tools to answer questions and perform tasks.\",\n advanced=False,\n ),\n *LCToolsAgentComponent._base_inputs,\n *memory_inputs,\n BoolInput(\n name=\"add_current_date_tool\",\n display_name=\"Current Date\",\n advanced=True,\n info=\"If true, will add a tool to the agent that returns the current date.\",\n value=True,\n ),\n ]\n outputs = [Output(name=\"response\", display_name=\"Response\", method=\"message_response\")]\n\n async def message_response(self) -> Message:\n try:\n # Get LLM model and validate\n llm_model, display_name = self.get_llm()\n if llm_model is None:\n msg = \"No language model selected. Please choose a model to proceed.\"\n raise ValueError(msg)\n self.model_name = get_model_name(llm_model, display_name=display_name)\n\n # Get memory data\n self.chat_history = await self.get_memory_data()\n\n # Add current date tool if enabled\n if self.add_current_date_tool:\n if not isinstance(self.tools, list): # type: ignore[has-type]\n self.tools = []\n current_date_tool = (await CurrentDateComponent(**self.get_base_args()).to_toolkit()).pop(0)\n if not isinstance(current_date_tool, StructuredTool):\n msg = \"CurrentDateComponent must be converted to a StructuredTool\"\n raise TypeError(msg)\n self.tools.append(current_date_tool)\n\n # Validate tools\n if not self.tools:\n msg = \"Tools are required to run the agent. Please add at least one tool.\"\n raise ValueError(msg)\n\n # Set up and run agent\n self.set(\n llm=llm_model,\n tools=self.tools,\n chat_history=self.chat_history,\n input_value=self.input_value,\n system_prompt=self.system_prompt,\n )\n agent = self.create_agent_runnable()\n return await self.run_agent(agent)\n\n except (ValueError, TypeError, KeyError) as e:\n logger.error(f\"{type(e).__name__}: {e!s}\")\n raise\n except ExceptionWithMessageError as e:\n logger.error(f\"ExceptionWithMessageError occurred: {e}\")\n raise\n except Exception as e:\n logger.error(f\"Unexpected error: {e!s}\")\n raise\n\n async def get_memory_data(self):\n memory_kwargs = {\n component_input.name: getattr(self, f\"{component_input.name}\") for component_input in self.memory_inputs\n }\n # filter out empty values\n memory_kwargs = {k: v for k, v in memory_kwargs.items() if v is not None}\n\n return await MemoryComponent(**self.get_base_args()).set(**memory_kwargs).retrieve_messages()\n\n def get_llm(self):\n if not isinstance(self.agent_llm, str):\n return self.agent_llm, None\n\n try:\n provider_info = MODEL_PROVIDERS_DICT.get(self.agent_llm)\n if not provider_info:\n msg = f\"Invalid model provider: {self.agent_llm}\"\n raise ValueError(msg)\n\n component_class = provider_info.get(\"component_class\")\n display_name = component_class.display_name\n inputs = provider_info.get(\"inputs\")\n prefix = provider_info.get(\"prefix\", \"\")\n\n return self._build_llm_model(component_class, inputs, prefix), display_name\n\n except Exception as e:\n logger.error(f\"Error building {self.agent_llm} language model: {e!s}\")\n msg = f\"Failed to initialize language model: {e!s}\"\n raise ValueError(msg) from e\n\n def _build_llm_model(self, component, inputs, prefix=\"\"):\n model_kwargs = {input_.name: getattr(self, f\"{prefix}{input_.name}\") for input_ in inputs}\n return component.set(**model_kwargs).build_model()\n\n def set_component_params(self, component):\n provider_info = MODEL_PROVIDERS_DICT.get(self.agent_llm)\n if provider_info:\n inputs = provider_info.get(\"inputs\")\n prefix = provider_info.get(\"prefix\")\n model_kwargs = {input_.name: getattr(self, f\"{prefix}{input_.name}\") for input_ in inputs}\n\n return component.set(**model_kwargs)\n return component\n\n def delete_fields(self, build_config: dotdict, fields: dict | list[str]) -> None:\n \"\"\"Delete specified fields from build_config.\"\"\"\n for field in fields:\n build_config.pop(field, None)\n\n def update_input_types(self, build_config: dotdict) -> dotdict:\n \"\"\"Update input types for all fields in build_config.\"\"\"\n for key, value in build_config.items():\n if isinstance(value, dict):\n if value.get(\"input_types\") is None:\n build_config[key][\"input_types\"] = []\n elif hasattr(value, \"input_types\") and value.input_types is None:\n value.input_types = []\n return build_config\n\n async def update_build_config(\n self, build_config: dotdict, field_value: str, field_name: str | None = None\n ) -> dotdict:\n # Iterate over all providers in the MODEL_PROVIDERS_DICT\n # Existing logic for updating build_config\n if field_name in (\"agent_llm\",):\n build_config[\"agent_llm\"][\"value\"] = field_value\n provider_info = MODEL_PROVIDERS_DICT.get(field_value)\n if provider_info:\n component_class = provider_info.get(\"component_class\")\n if component_class and hasattr(component_class, \"update_build_config\"):\n # Call the component class's update_build_config method\n build_config = await update_component_build_config(\n component_class, build_config, field_value, \"model_name\"\n )\n\n provider_configs: dict[str, tuple[dict, list[dict]]] = {\n provider: (\n MODEL_PROVIDERS_DICT[provider][\"fields\"],\n [\n MODEL_PROVIDERS_DICT[other_provider][\"fields\"]\n for other_provider in MODEL_PROVIDERS_DICT\n if other_provider != provider\n ],\n )\n for provider in MODEL_PROVIDERS_DICT\n }\n if field_value in provider_configs:\n fields_to_add, fields_to_delete = provider_configs[field_value]\n\n # Delete fields from other providers\n for fields in fields_to_delete:\n self.delete_fields(build_config, fields)\n\n # Add provider-specific fields\n if field_value == \"OpenAI\" and not any(field in build_config for field in fields_to_add):\n build_config.update(fields_to_add)\n else:\n build_config.update(fields_to_add)\n # Reset input types for agent_llm\n build_config[\"agent_llm\"][\"input_types\"] = []\n elif field_value == \"Custom\":\n # Delete all provider fields\n self.delete_fields(build_config, ALL_PROVIDER_FIELDS)\n # Update with custom component\n custom_component = DropdownInput(\n name=\"agent_llm\",\n display_name=\"Language Model\",\n options=[*sorted(MODEL_PROVIDERS_DICT.keys()), \"Custom\"],\n value=\"Custom\",\n real_time_refresh=True,\n input_types=[\"LanguageModel\"],\n options_metadata=[MODELS_METADATA[key] for key in sorted(MODELS_METADATA.keys())]\n + [{\"icon\": \"brain\"}],\n )\n build_config.update({\"agent_llm\": custom_component.to_dict()})\n # Update input types for all fields\n build_config = self.update_input_types(build_config)\n\n # Validate required keys\n default_keys = [\n \"code\",\n \"_type\",\n \"agent_llm\",\n \"tools\",\n \"input_value\",\n \"add_current_date_tool\",\n \"system_prompt\",\n \"agent_description\",\n \"max_iterations\",\n \"handle_parsing_errors\",\n \"verbose\",\n ]\n missing_keys = [key for key in default_keys if key not in build_config]\n if missing_keys:\n msg = f\"Missing required keys in build_config: {missing_keys}\"\n raise ValueError(msg)\n if (\n isinstance(self.agent_llm, str)\n and self.agent_llm in MODEL_PROVIDERS_DICT\n and field_name in MODEL_DYNAMIC_UPDATE_FIELDS\n ):\n provider_info = MODEL_PROVIDERS_DICT.get(self.agent_llm)\n if provider_info:\n component_class = provider_info.get(\"component_class\")\n component_class = self.set_component_params(component_class)\n prefix = provider_info.get(\"prefix\")\n if component_class and hasattr(component_class, \"update_build_config\"):\n # Call each component class's update_build_config method\n # remove the prefix from the field_name\n if isinstance(field_name, str) and isinstance(prefix, str):\n field_name = field_name.replace(prefix, \"\")\n build_config = await update_component_build_config(\n component_class, build_config, field_value, \"model_name\"\n )\n return dotdict({k: v.to_dict() if hasattr(v, \"to_dict\") else v for k, v in build_config.items()})\n" }, "handle_parsing_errors": { "_input_type": "BoolInput", "advanced": true, "display_name": "Handle Parse Errors", "dynamic": false, "info": "Should the Agent fix errors when reading user input for better processing?", "input_types": [], "list": false, "list_add_label": "Add More", "name": "handle_parsing_errors", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true }, "input_value": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input", "dynamic": false, "info": "The input provided by the user for the agent to process.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "input_value", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "max_iterations": { "_input_type": "IntInput", "advanced": true, "display_name": "Max Iterations", "dynamic": false, "info": "The maximum number of attempts the agent can make to complete its task before it stops.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "max_iterations", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "int", "value": 15 }, "max_output_tokens": { "_input_type": "IntInput", "advanced": false, "display_name": "Max Output Tokens", "dynamic": false, "info": "The maximum number of tokens to generate.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "max_output_tokens", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "int", "value": "" }, "memory": { "_input_type": "HandleInput", "advanced": true, "display_name": "External Memory", "dynamic": false, "info": "Retrieve messages from an external memory. If empty, it will use the Langflow tables.", "input_types": [ "Memory" ], "list": false, "list_add_label": "Add More", "name": "memory", "placeholder": "", "required": false, "show": true, "title_case": false, "trace_as_metadata": true, "type": "other", "value": "" }, "model_name": { "_input_type": "DropdownInput", "advanced": false, "combobox": true, "dialog_inputs": {}, "display_name": "Model", "dynamic": false, "info": "To see the model names, first choose a provider. Then, enter your API key and click the refresh button next to the model name.", "input_types": [], "name": "model_name", "options": [ "learnlm-2.0-flash-experimental", "gemma-3n-e4b-it", "gemma-3-4b-it", "gemma-3-27b-it", "gemma-3-1b-it", "gemma-3-12b-it", "gemini-pro-vision", "gemini-exp-1206", "gemini-2.5-pro-preview-tts", "gemini-2.5-pro-preview-05-06", "gemini-2.5-pro-preview-03-25", "gemini-2.5-pro-exp-03-25", "gemini-2.5-flash-preview-tts", "gemini-2.5-flash-preview-05-20", "gemini-2.5-flash-preview-04-17-thinking", "gemini-2.5-flash-preview-04-17", "gemini-2.0-pro-exp-02-05", "gemini-2.0-pro-exp", "gemini-2.0-flash-thinking-exp-1219", "gemini-2.0-flash-thinking-exp-01-21", "gemini-2.0-flash-thinking-exp", "gemini-2.0-flash-preview-image-generation", "gemini-2.0-flash-lite-preview-02-05", "gemini-2.0-flash-lite-preview", "gemini-2.0-flash-lite-001", "gemini-2.0-flash-lite", "gemini-2.0-flash-exp-image-generation", "gemini-2.0-flash-exp", "gemini-2.0-flash-001", "gemini-2.0-flash", "gemini-1.5-pro-latest", "gemini-1.5-pro-002", "gemini-1.5-pro-001", "gemini-1.5-pro", "gemini-1.5-flash-latest", "gemini-1.5-flash-8b-latest", "gemini-1.5-flash-8b-exp-0924", "gemini-1.5-flash-8b-exp-0827", "gemini-1.5-flash-8b-001", "gemini-1.5-flash-8b", "gemini-1.5-flash-002", "gemini-1.5-flash-001-tuning", "gemini-1.5-flash-001", "gemini-1.5-flash", "gemini-1.0-pro-vision-latest" ], "options_metadata": [], "placeholder": "", "real_time_refresh": false, "refresh_button": true, "required": false, "show": true, "title_case": false, "toggle": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "learnlm-2.0-flash-experimental" }, "n": { "_input_type": "IntInput", "advanced": true, "display_name": "N", "dynamic": false, "info": "Number of chat completions to generate for each prompt. Note that the API may not return the full n completions if duplicates are generated.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "n", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "int", "value": "" }, "n_messages": { "_input_type": "IntInput", "advanced": true, "display_name": "Number of Messages", "dynamic": false, "info": "Number of messages to retrieve.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "n_messages", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "int", "value": 100 }, "order": { "_input_type": "DropdownInput", "advanced": true, "combobox": false, "dialog_inputs": {}, "display_name": "Order", "dynamic": false, "info": "Order of the messages.", "input_types": [], "name": "order", "options": [ "Ascending", "Descending" ], "options_metadata": [], "placeholder": "", "required": false, "show": true, "title_case": false, "toggle": false, "tool_mode": true, "trace_as_metadata": true, "type": "str", "value": "Ascending" }, "sender": { "_input_type": "DropdownInput", "advanced": true, "combobox": false, "dialog_inputs": {}, "display_name": "Sender Type", "dynamic": false, "info": "Filter by sender type.", "input_types": [], "name": "sender", "options": [ "Machine", "User", "Machine and User" ], "options_metadata": [], "placeholder": "", "required": false, "show": true, "title_case": false, "toggle": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "Machine and User" }, "sender_name": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Sender Name", "dynamic": false, "info": "Filter by sender name.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "sender_name", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "session_id": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Session ID", "dynamic": false, "info": "The session ID of the chat. If empty, the current session ID parameter will be used.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "session_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "system_prompt": { "_input_type": "MultilineInput", "advanced": false, "copy_field": false, "display_name": "Agent Instructions", "dynamic": false, "info": "System Prompt: Initial instructions and context provided to guide the agent's behavior.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "multiline": true, "name": "system_prompt", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "You are a helpful video editor assistance that has access to custom fuctions to interact with twelvelabs api and help create index, upload video and perform semantic search on the videos on index." }, "temperature": { "_input_type": "SliderInput", "advanced": true, "display_name": "Temperature", "dynamic": false, "info": "Controls randomness. Lower values are more deterministic, higher values are more creative.", "input_types": [], "max_label": "", "max_label_icon": "", "min_label": "", "min_label_icon": "", "name": "temperature", "placeholder": "", "range_spec": { "max": 2, "min": 0, "step": 0.01, "step_type": "float" }, "required": false, "show": true, "slider_buttons": false, "slider_buttons_options": [], "slider_input": false, "title_case": false, "tool_mode": false, "type": "slider", "value": 0.1 }, "template": { "_input_type": "MultilineInput", "advanced": true, "copy_field": false, "display_name": "Template", "dynamic": false, "info": "The template to use for formatting the data. It can contain the keys {text}, {sender} or any other key in the message data.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "multiline": true, "name": "template", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "{sender_name}: {text}" }, "tool_model_enabled": { "_input_type": "BoolInput", "advanced": true, "display_name": "Tool Model Enabled", "dynamic": false, "info": "Whether to use the tool model.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "tool_model_enabled", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": false }, "tools": { "_input_type": "HandleInput", "advanced": false, "display_name": "Tools", "dynamic": false, "info": "These are the tools that the agent can use to help with tasks.", "input_types": [ "Tool" ], "list": true, "list_add_label": "Add More", "name": "tools", "placeholder": "", "required": false, "show": true, "title_case": false, "trace_as_metadata": true, "type": "other", "value": "" }, "top_k": { "_input_type": "IntInput", "advanced": true, "display_name": "Top K", "dynamic": false, "info": "Decode using top-k sampling: consider the set of top_k most probable tokens. Must be positive.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "top_k", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "int", "value": "" }, "top_p": { "_input_type": "FloatInput", "advanced": true, "display_name": "Top P", "dynamic": false, "info": "The maximum cumulative probability of tokens to consider when sampling.", "input_types": [], "list": false, "list_add_label": "Add More", "name": "top_p", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "float", "value": "" }, "verbose": { "_input_type": "BoolInput", "advanced": true, "display_name": "Verbose", "dynamic": false, "info": "", "input_types": [], "list": false, "list_add_label": "Add More", "name": "verbose", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true } }, "tool_mode": false }, "showNode": true, "type": "Agent" }, "dragging": false, "id": "Agent-W6zzd", "measured": { "height": 707, "width": 320 }, "position": { "x": 1198.4876601573922, "y": -85.81763214607481 }, "selected": false, "type": "genericNode" }, { "data": { "id": "ChatInput-BLgCA", "node": { "base_classes": [ "Message" ], "beta": false, "category": "inputs", "conditional_paths": [], "custom_fields": {}, "description": "Get chat inputs from the Playground.", "display_name": "Chat Input", "documentation": "", "edited": false, "field_order": [ "input_value", "should_store_message", "sender", "sender_name", "session_id", "files", "background_color", "chat_icon", "text_color" ], "frozen": false, "icon": "MessagesSquare", "key": "ChatInput", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": true, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Message", "method": "message_response", "name": "message", "selected": "Message", "tool_mode": true, "types": [ "Message" ], "value": "__UNDEFINED__" } ], "pinned": false, "score": 0.0020353564437605998, "template": { "_type": "Component", "background_color": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Background Color", "dynamic": false, "info": "The background color of the icon.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "background_color", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "chat_icon": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Icon", "dynamic": false, "info": "The icon of the message.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "chat_icon", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "from langflow.base.data.utils import IMG_FILE_TYPES, TEXT_FILE_TYPES\nfrom langflow.base.io.chat import ChatComponent\nfrom langflow.inputs import BoolInput\nfrom langflow.io import (\n DropdownInput,\n FileInput,\n MessageTextInput,\n MultilineInput,\n Output,\n)\nfrom langflow.schema.message import Message\nfrom langflow.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_USER,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatInput(ChatComponent):\n display_name = \"Chat Input\"\n description = \"Get chat inputs from the Playground.\"\n icon = \"MessagesSquare\"\n name = \"ChatInput\"\n minimized = True\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n value=\"\",\n info=\"Message to be passed as input.\",\n input_types=[],\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_USER,\n info=\"Type of sender.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_USER,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n FileInput(\n name=\"files\",\n display_name=\"Files\",\n file_types=TEXT_FILE_TYPES + IMG_FILE_TYPES,\n info=\"Files to be sent with the message.\",\n advanced=True,\n is_list=True,\n temp_file=True,\n ),\n MessageTextInput(\n name=\"background_color\",\n display_name=\"Background Color\",\n info=\"The background color of the icon.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"chat_icon\",\n display_name=\"Icon\",\n info=\"The icon of the message.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"text_color\",\n display_name=\"Text Color\",\n info=\"The text color of the name\",\n advanced=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Message\", name=\"message\", method=\"message_response\"),\n ]\n\n async def message_response(self) -> Message:\n background_color = self.background_color\n text_color = self.text_color\n icon = self.chat_icon\n\n message = await Message.create(\n text=self.input_value,\n sender=self.sender,\n sender_name=self.sender_name,\n session_id=self.session_id,\n files=self.files,\n properties={\n \"background_color\": background_color,\n \"text_color\": text_color,\n \"icon\": icon,\n },\n )\n if self.session_id and isinstance(message, Message) and self.should_store_message:\n stored_message = await self.send_message(\n message,\n )\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n" }, "files": { "_input_type": "FileInput", "advanced": true, "display_name": "Files", "dynamic": false, "fileTypes": [ "txt", "md", "mdx", "csv", "json", "yaml", "yml", "xml", "html", "htm", "pdf", "docx", "py", "sh", "sql", "js", "ts", "tsx", "jpg", "jpeg", "png", "bmp", "image" ], "file_path": "", "info": "Files to be sent with the message.", "list": true, "list_add_label": "Add More", "name": "files", "placeholder": "", "required": false, "show": true, "title_case": false, "trace_as_metadata": true, "type": "file", "value": "" }, "input_value": { "_input_type": "MultilineInput", "advanced": false, "copy_field": false, "display_name": "Text", "dynamic": false, "info": "Message to be passed as input.", "input_types": [], "list": false, "list_add_label": "Add More", "load_from_db": false, "multiline": true, "name": "input_value", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "sender": { "_input_type": "DropdownInput", "advanced": true, "combobox": false, "dialog_inputs": {}, "display_name": "Sender Type", "dynamic": false, "info": "Type of sender.", "name": "sender", "options": [ "Machine", "User" ], "options_metadata": [], "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "User" }, "sender_name": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Sender Name", "dynamic": false, "info": "Name of the sender.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "sender_name", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "User" }, "session_id": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Session ID", "dynamic": false, "info": "The session ID of the chat. If empty, the current session ID parameter will be used.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "session_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "should_store_message": { "_input_type": "BoolInput", "advanced": true, "display_name": "Store Messages", "dynamic": false, "info": "Store the message in the history.", "list": false, "list_add_label": "Add More", "name": "should_store_message", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true }, "text_color": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Text Color", "dynamic": false, "info": "The text color of the name", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "text_color", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" } }, "tool_mode": false }, "showNode": false, "type": "ChatInput" }, "dragging": false, "id": "ChatInput-BLgCA", "measured": { "height": 66, "width": 192 }, "position": { "x": 929.6002617923111, "y": 982.8701931597282 }, "selected": false, "type": "genericNode" }, { "data": { "id": "ChatOutput-Cm5t9", "node": { "base_classes": [ "Message" ], "beta": false, "category": "outputs", "conditional_paths": [], "custom_fields": {}, "description": "Display a chat message in the Playground.", "display_name": "Chat Output", "documentation": "", "edited": false, "field_order": [ "input_value", "should_store_message", "sender", "sender_name", "session_id", "data_template", "background_color", "chat_icon", "text_color", "clean_data" ], "frozen": false, "icon": "MessagesSquare", "key": "ChatOutput", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": true, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Message", "method": "message_response", "name": "message", "selected": "Message", "tool_mode": true, "types": [ "Message" ], "value": "__UNDEFINED__" } ], "pinned": false, "score": 0.003169567463043492, "template": { "_type": "Component", "background_color": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Background Color", "dynamic": false, "info": "The background color of the icon.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "background_color", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "chat_icon": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Icon", "dynamic": false, "info": "The icon of the message.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "chat_icon", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "clean_data": { "_input_type": "BoolInput", "advanced": true, "display_name": "Basic Clean Data", "dynamic": false, "info": "Whether to clean the data", "list": false, "list_add_label": "Add More", "name": "clean_data", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true }, "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "from collections.abc import Generator\nfrom typing import Any\n\nfrom langflow.base.io.chat import ChatComponent\nfrom langflow.inputs import BoolInput\nfrom langflow.inputs.inputs import HandleInput\nfrom langflow.io import DropdownInput, MessageTextInput, Output\nfrom langflow.schema.data import Data\nfrom langflow.schema.dataframe import DataFrame\nfrom langflow.schema.message import Message\nfrom langflow.schema.properties import Source\nfrom langflow.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_AI,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatOutput(ChatComponent):\n display_name = \"Chat Output\"\n description = \"Display a chat message in the Playground.\"\n icon = \"MessagesSquare\"\n name = \"ChatOutput\"\n minimized = True\n\n inputs = [\n HandleInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Message to be passed as output.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_AI,\n advanced=True,\n info=\"Type of sender.\",\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_AI,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"data_template\",\n display_name=\"Data Template\",\n value=\"{text}\",\n advanced=True,\n info=\"Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.\",\n ),\n MessageTextInput(\n name=\"background_color\",\n display_name=\"Background Color\",\n info=\"The background color of the icon.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"chat_icon\",\n display_name=\"Icon\",\n info=\"The icon of the message.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"text_color\",\n display_name=\"Text Color\",\n info=\"The text color of the name\",\n advanced=True,\n ),\n BoolInput(\n name=\"clean_data\",\n display_name=\"Basic Clean Data\",\n value=True,\n info=\"Whether to clean the data\",\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Message\",\n name=\"message\",\n method=\"message_response\",\n ),\n ]\n\n def _build_source(self, id_: str | None, display_name: str | None, source: str | None) -> Source:\n source_dict = {}\n if id_:\n source_dict[\"id\"] = id_\n if display_name:\n source_dict[\"display_name\"] = display_name\n if source:\n # Handle case where source is a ChatOpenAI object\n if hasattr(source, \"model_name\"):\n source_dict[\"source\"] = source.model_name\n elif hasattr(source, \"model\"):\n source_dict[\"source\"] = str(source.model)\n else:\n source_dict[\"source\"] = str(source)\n return Source(**source_dict)\n\n async def message_response(self) -> Message:\n # First convert the input to string if needed\n text = self.convert_to_string()\n # Get source properties\n source, icon, display_name, source_id = self.get_properties_from_source_component()\n background_color = self.background_color\n text_color = self.text_color\n if self.chat_icon:\n icon = self.chat_icon\n\n # Create or use existing Message object\n if isinstance(self.input_value, Message):\n message = self.input_value\n # Update message properties\n message.text = text\n else:\n message = Message(text=text)\n\n # Set message properties\n message.sender = self.sender\n message.sender_name = self.sender_name\n message.session_id = self.session_id\n message.flow_id = self.graph.flow_id if hasattr(self, \"graph\") else None\n message.properties.source = self._build_source(source_id, display_name, source)\n message.properties.icon = icon\n message.properties.background_color = background_color\n message.properties.text_color = text_color\n\n # Store message if needed\n if self.session_id and self.should_store_message:\n stored_message = await self.send_message(message)\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n\n def _validate_input(self) -> None:\n \"\"\"Validate the input data and raise ValueError if invalid.\"\"\"\n if self.input_value is None:\n msg = \"Input data cannot be None\"\n raise ValueError(msg)\n if isinstance(self.input_value, list) and not all(\n isinstance(item, Message | Data | DataFrame | str) for item in self.input_value\n ):\n invalid_types = [\n type(item).__name__\n for item in self.input_value\n if not isinstance(item, Message | Data | DataFrame | str)\n ]\n msg = f\"Expected Data or DataFrame or Message or str, got {invalid_types}\"\n raise TypeError(msg)\n if not isinstance(\n self.input_value,\n Message | Data | DataFrame | str | list | Generator | type(None),\n ):\n type_name = type(self.input_value).__name__\n msg = f\"Expected Data or DataFrame or Message or str, Generator or None, got {type_name}\"\n raise TypeError(msg)\n\n def _safe_convert(self, data: Any) -> str:\n \"\"\"Safely convert input data to string.\"\"\"\n try:\n if isinstance(data, str):\n return data\n if isinstance(data, Message):\n return data.get_text()\n if isinstance(data, Data):\n if data.get_text() is None:\n msg = \"Empty Data object\"\n raise ValueError(msg)\n return data.get_text()\n if isinstance(data, DataFrame):\n if self.clean_data:\n # Remove empty rows\n data = data.dropna(how=\"all\")\n # Remove empty lines in each cell\n data = data.replace(r\"^\\s*$\", \"\", regex=True)\n # Replace multiple newlines with a single newline\n data = data.replace(r\"\\n+\", \"\\n\", regex=True)\n\n # Replace pipe characters to avoid markdown table issues\n processed_data = data.replace(r\"\\|\", r\"\\\\|\", regex=True)\n\n processed_data = processed_data.map(\n lambda x: str(x).replace(\"\\n\", \"<br/>\") if isinstance(x, str) else x\n )\n\n return processed_data.to_markdown(index=False)\n return str(data)\n except (ValueError, TypeError, AttributeError) as e:\n msg = f\"Error converting data: {e!s}\"\n raise ValueError(msg) from e\n\n def convert_to_string(self) -> str | Generator[Any, None, None]:\n \"\"\"Convert input data to string with proper error handling.\"\"\"\n self._validate_input()\n if isinstance(self.input_value, list):\n return \"\\n\".join([self._safe_convert(item) for item in self.input_value])\n if isinstance(self.input_value, Generator):\n return self.input_value\n return self._safe_convert(self.input_value)\n" }, "data_template": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Data Template", "dynamic": false, "info": "Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "data_template", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "{text}" }, "input_value": { "_input_type": "HandleInput", "advanced": false, "display_name": "Text", "dynamic": false, "info": "Message to be passed as output.", "input_types": [ "Data", "DataFrame", "Message" ], "list": false, "list_add_label": "Add More", "name": "input_value", "placeholder": "", "required": true, "show": true, "title_case": false, "trace_as_metadata": true, "type": "other", "value": "" }, "sender": { "_input_type": "DropdownInput", "advanced": true, "combobox": false, "dialog_inputs": {}, "display_name": "Sender Type", "dynamic": false, "info": "Type of sender.", "name": "sender", "options": [ "Machine", "User" ], "options_metadata": [], "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "Machine" }, "sender_name": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Sender Name", "dynamic": false, "info": "Name of the sender.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "sender_name", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "AI" }, "session_id": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Session ID", "dynamic": false, "info": "The session ID of the chat. If empty, the current session ID parameter will be used.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "session_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "should_store_message": { "_input_type": "BoolInput", "advanced": true, "display_name": "Store Messages", "dynamic": false, "info": "Store the message in the history.", "list": false, "list_add_label": "Add More", "name": "should_store_message", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "bool", "value": true }, "text_color": { "_input_type": "MessageTextInput", "advanced": true, "display_name": "Text Color", "dynamic": false, "info": "The text color of the name", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "text_color", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" } }, "tool_mode": false }, "showNode": false, "type": "ChatOutput" }, "dragging": false, "id": "ChatOutput-Cm5t9", "measured": { "height": 66, "width": 192 }, "position": { "x": 1970.6739099748165, "y": 966.5358778964039 }, "selected": true, "type": "genericNode" }, { "data": { "id": "CreateSearchIndex-ynUMz", "node": { "base_classes": [ "Text" ], "beta": false, "conditional_paths": [], "custom_fields": {}, "description": "Use this component to create a index in twelvelabs for video sematic search.", "display_name": "CreateIndex", "documentation": "https://docs.langflow.org/components-custom-components", "edited": true, "field_order": [ "index_name" ], "frozen": false, "icon": "code", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Toolset", "hidden": false, "method": "to_toolkit", "name": "component_as_tool", "options": null, "required_inputs": null, "selected": "Tool", "tool_mode": true, "types": [ "Tool" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { "_type": "Component", "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "# from langflow.field_typing import Data\nfrom langflow.custom import Component\nfrom langflow.io import MessageTextInput, Output\nfrom langflow.schema import Data\n\n\nclass CreateIndex(Component):\n display_name = \"CreateIndex\"\n description = \"Use this component to create a index in twelvelabs for video sematic search.\"\n documentation: str = \"https://docs.langflow.org/components-custom-components\"\n icon = \"code\"\n name = \"CreateIndex\"\n\n inputs = [\n MessageTextInput(\n name=\"index_name\",\n display_name=\"Input your index name\",\n info=\"This component is used to create a index before we can upload a video and search on it. It return the index_id.\",\n tool_mode=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Output\", name=\"output\", method=\"build_index\"),\n ]\n\n def build_index(self) -> str:\n \n API_KEY = \"tlk_3NC3Q3Y0MQTHG82PQKQGV3T1YB9S\"\n INDEX_NAME = self.index_name\n ENGINE = \"marengo2.7\"\n OPTIONS = [\"visual\", \"audio\"]\n BASE_URL = \"https://api.twelvelabs.io/v1.3\"\n \n headers = {\n \"x-api-key\": API_KEY,\n \"accept\": \"application/json\",\n \"content-type\": \"application/json\"\n }\n\n create_index_payload = {\n \"index_name\": INDEX_NAME,\n \"models\": [\n {\n \"model_name\": ENGINE,\n \"model_options\": OPTIONS\n \n },\n {\n \"model_name\": \"pegasus1.2\",\n \"model_options\": OPTIONS\n }\n ]\n }\n res = requests.post(f\"{BASE_URL}/indexes\", json=create_index_payload, headers=headers)\n res.raise_for_status()\n index = res.json()\n index_id = index[\"_id\"]\n out = f\"Created index: id={index_id}\"\n \n self.status = Data(data=out)\n \n return out\n\n \n \n\n" }, "index_name": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input your index name", "dynamic": false, "info": "This component is used to create a index before we can upload a video and search on it. It return the index_id.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "index_name", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "tools_metadata": { "_input_type": "ToolsInput", "advanced": false, "display_name": "Actions", "dynamic": false, "info": "Modify tool names and descriptions to help agents understand when to use each tool.", "is_list": true, "list_add_label": "Add More", "name": "tools_metadata", "placeholder": "", "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "tools", "value": [ { "args": { "index_name": { "default": "", "description": "This component is used to create a index before we can upload a video and search on it. It return the index_id.", "title": "Index Name", "type": "string" } }, "description": "CreateIndex. build_index - Use this component to create a index in twelvelabs for video sematic search.", "display_description": "CreateIndex. build_index - Use this component to create a index in twelvelabs for video sematic search.", "display_name": "build_index", "name": "build_index", "readonly": false, "status": true, "tags": [ "build_index" ] } ] } }, "tool_mode": true }, "showNode": true, "type": "CreateIndex" }, "dragging": false, "id": "CreateSearchIndex-ynUMz", "measured": { "height": 248, "width": 320 }, "position": { "x": 394.8799936252261, "y": 583.1694527201979 }, "selected": false, "type": "genericNode" }, { "data": { "id": "UploadVideo-RxJXM", "node": { "base_classes": [ "Data" ], "beta": false, "conditional_paths": [], "custom_fields": {}, "description": "Use this component to upload a video on a index given the video url and index id in twelvelabs for video sematic search.", "display_name": "UploadVideo", "documentation": "https://docs.langflow.org/components-custom-components", "edited": true, "field_order": [ "video_url", "index_id" ], "frozen": false, "icon": "code", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Toolset", "hidden": false, "method": "to_toolkit", "name": "component_as_tool", "options": null, "required_inputs": null, "selected": "Tool", "tool_mode": true, "types": [ "Tool" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { "_type": "Component", "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "# from langflow.field_typing import Data\nfrom langflow.custom import Component\nfrom langflow.io import MessageTextInput, Output\nfrom langflow.schema import Data\n\n\nclass UploadVideo(Component):\n display_name = \"UploadVideo\"\n description = \"Use this component to upload a video on a index given the video url and index id in twelvelabs for video sematic search.\"\n documentation: str = \"https://docs.langflow.org/components-custom-components\"\n icon = \"code\"\n name = \"UploadVideo\"\n\n inputs = [\n MessageTextInput(\n name=\"video_url\",\n display_name=\"Input your video url\",\n info=\"This component is used to get video url to be uploaded after we have created a index and search on it. It return the video id.\",\n tool_mode=True,\n ),\n MessageTextInput(\n name=\"index_id\",\n display_name=\"Input your index id\",\n info=\"This component is used to get index id on which to upload the video after we have created a index and search on it. It return the video id.\",\n tool_mode=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Output\", name=\"output\", method=\"upload_video\"),\n ]\n \n def download(video_name):\n\n url = \"https://cdn.pixabay.com/video/2024/09/21/232538_large.mp4\"\n filename = \"downloaded_video.mp4\"\n \n response = requests.get(url, stream=True)\n response.raise_for_status()\n \n with open(filename, \"wb\") as f:\n for chunk in response.iter_content(chunk_size=8192):\n if chunk:\n f.write(chunk)\n \n return filename\n\n\n def upload_video(self) -> Data:\n \n API_KEY = \"tlk_3NC3Q3Y0MQTHG82PQKQGV3T1YB9S\"\n INDEX_ID = self.index_id\n BASE_URL = \"https://api.twelvelabs.io/v1.3\"\n VIDEO_LOC = self.video_url #download(self.video_name)\n \n headers = {\n \"x-api-key\": API_KEY #,\n #\"content-type\": \"application/json\",\n }\n\n create_index_payload = {\n \"index_id\": INDEX_ID,\n \"video_url\": VIDEO_LOC\n }\n res = requests.post(f\"{BASE_URL}/tasks\", json=create_index_payload, headers=headers)\n res.raise_for_status()\n index = res.json()\n video_id = index[\"_id\"]\n out = f\"Created video: id={video_id}\"\n \n self.status = Data(out)\n \n return video_id" }, "index_id": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input your index id", "dynamic": false, "info": "This component is used to get index id on which to upload the video after we have created a index and search on it. It return the video id.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "index_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "tools_metadata": { "_input_type": "ToolsInput", "advanced": false, "display_name": "Actions", "dynamic": false, "info": "Modify tool names and descriptions to help agents understand when to use each tool.", "is_list": true, "list_add_label": "Add More", "name": "tools_metadata", "placeholder": "", "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "tools", "value": [ { "args": { "index_id": { "default": "", "description": "This component is used to get index id on which to upload the video after we have created a index and search on it. It return the video id.", "title": "Index Id", "type": "string" }, "video_url": { "default": "", "description": "This component is used to get video url to be uploaded after we have created a index and search on it. It return the video id.", "title": "Video Url", "type": "string" } }, "description": "UploadVideo. upload_video - Use this component to upload a video on a index given the video url and index id in twelvelabs for video sematic search.", "display_description": "UploadVideo. upload_video - Use this component to upload a video on a index given the video url and index id in twelvelabs for video sematic search.", "display_name": "upload_video", "name": "upload_video", "readonly": false, "status": true, "tags": [ "upload_video" ] } ] }, "video_url": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input your video url", "dynamic": false, "info": "This component is used to get video url to be uploaded after we have created a index and search on it. It return the video id.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "video_url", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" } }, "tool_mode": true }, "showNode": true, "type": "UploadVideo" }, "dragging": false, "id": "UploadVideo-RxJXM", "measured": { "height": 268, "width": 320 }, "position": { "x": 350.29407406935627, "y": 898.9174224987505 }, "selected": false, "type": "genericNode" }, { "data": { "id": "SearchOnVideo-LY0Qv", "node": { "base_classes": [ "Data" ], "beta": false, "conditional_paths": [], "custom_fields": {}, "description": "Use this component to perform sematic search on uploaded video in twelvelabs.", "display_name": "SearchOnVideo", "documentation": "https://docs.langflow.org/components-custom-components", "edited": true, "field_order": [ "index_id", "query_text" ], "frozen": false, "icon": "code", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Toolset", "hidden": null, "method": "to_toolkit", "name": "component_as_tool", "options": null, "required_inputs": null, "selected": "Tool", "tool_mode": true, "types": [ "Tool" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { "_type": "Component", "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "# from langflow.field_typing import Data\nfrom langflow.custom import Component\nfrom langflow.io import MessageTextInput, Output\nfrom langflow.schema import Data\n\n\nclass SearchOnVideo(Component):\n display_name = \"SearchOnVideo\"\n description = \"Use this component to perform sematic search on uploaded video in twelvelabs.\"\n documentation: str = \"https://docs.langflow.org/components-custom-components\"\n icon = \"code\"\n name = \"SearchOnVideo\"\n\n inputs = [\n MessageTextInput(\n name=\"index_id\",\n display_name=\"Input your index id\",\n info=\"The index id to query on\",\n tool_mode=True,\n ),\n MessageTextInput(\n name=\"query_text\",\n display_name=\"Input your query text\",\n info=\"The query text for the index videos\",\n tool_mode=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Output\", name=\"output\", method=\"search_video\"),\n ]\n\n def search_video(self) -> Data:\n \n API_KEY = \"tlk_3NC3Q3Y0MQTHG82PQKQGV3T1YB9S\"\n index_id = self.index_id\n ENGINE = \"marengo2.7\"\n OPTIONS = [\"visual\", \"audio\"]\n BASE_URL = \"https://api.twelvelabs.io/v1.3\"\n \n headers = {\n \"x-api-key\": API_KEY\n #\"accept\": \"application/json\",\n #\"content-type\": \"application/json\",\n }\n \n create_index_payload = {\n \"query_text\": self.query_text,\n \"index_id\": index_id,\n \"search_options\": (\"visual\"),\n \"adjust_confidence_level\": \"0.5\",\n \"group_by\": \"clip\",\n \"sort_option\": \"score\",\n \"operator\": \"or\",\n \"page_limit\": \"10\"\n }\n \n res = requests.post(f\"{BASE_URL}/search\", files=create_index_payload, data=create_index_payload, headers=headers)\n #res.raise_for_status()\n out = res.json()\n #index_id = index[\"_id\"]\n \n #out = f\"Created index: id={index_id} name={index['name']}\"\n \n self.status = Data(value=out)\n \n return self.status\n \n \n\n" }, "index_id": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input your index id", "dynamic": false, "info": "The index id to query on", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "index_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "query_text": { "_input_type": "MessageTextInput", "advanced": false, "display_name": "Input your query text", "dynamic": false, "info": "The query text for the index videos", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "query_text", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": true, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "tools_metadata": { "_input_type": "ToolsInput", "advanced": false, "display_name": "Actions", "dynamic": false, "info": "Modify tool names and descriptions to help agents understand when to use each tool.", "is_list": true, "list_add_label": "Add More", "name": "tools_metadata", "placeholder": "", "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "type": "tools", "value": [ { "args": { "index_id": { "default": "", "description": "The index id to query on", "title": "Index Id", "type": "string" }, "query_text": { "default": "", "description": "The query text for the index videos", "title": "Query Text", "type": "string" } }, "description": "SearchOnVideo. search_video - Use this component to perform sematic search on uploaded video in twelvelabs.", "display_description": "SearchOnVideo. search_video - Use this component to perform sematic search on uploaded video in twelvelabs.", "display_name": "search_video", "name": "search_video", "readonly": false, "status": true, "tags": [ "search_video" ] } ] } }, "tool_mode": true }, "showNode": true, "type": "SearchOnVideo" }, "dragging": false, "id": "SearchOnVideo-LY0Qv", "measured": { "height": 248, "width": 320 }, "position": { "x": 605.9699807110773, "y": 210.77119562223552 }, "selected": false, "type": "genericNode" }, { "data": { "id": "TwelveLabsPegasus-8YZUt", "node": { "base_classes": [ "Message" ], "beta": false, "conditional_paths": [], "custom_fields": {}, "description": "Chat with videos using Twelve Labs Pegasus API.", "display_name": "Twelve Labs Pegasus", "documentation": "https://github.com/twelvelabs-io/twelvelabs-developer-experience/blob/main/integrations/Langflow/TWELVE_LABS_COMPONENTS_README.md", "edited": false, "field_order": [ "videodata", "api_key", "video_id", "index_name", "index_id", "model_name", "message", "temperature" ], "frozen": false, "icon": "TwelveLabs", "legacy": false, "lf_version": "1.4.2", "metadata": {}, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, "display_name": "Message", "hidden": false, "method": "process_video", "name": "response", "selected": "Message", "tool_mode": true, "types": [ "Message" ], "value": "__UNDEFINED__" }, { "allows_loop": false, "cache": true, "display_name": "Video ID", "method": "get_video_id", "name": "processed_video_id", "selected": "Message", "tool_mode": true, "types": [ "Message" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { "_type": "Component", "api_key": { "_input_type": "SecretStrInput", "advanced": false, "display_name": "Twelve Labs API Key", "dynamic": false, "info": "Enter your Twelve Labs API Key.", "input_types": [], "load_from_db": false, "name": "api_key", "password": true, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "str", "value": "" }, "code": { "advanced": true, "dynamic": true, "fileTypes": [], "file_path": "", "info": "", "list": false, "load_from_db": false, "multiline": true, "name": "code", "password": false, "placeholder": "", "required": true, "show": true, "title_case": false, "type": "code", "value": "import json\nimport subprocess\nimport time\nfrom pathlib import Path\nfrom typing import Any\n\nfrom tenacity import retry, stop_after_attempt, wait_exponential\nfrom twelvelabs import TwelveLabs\n\nfrom langflow.custom import Component\nfrom langflow.field_typing.range_spec import RangeSpec\nfrom langflow.inputs import DataInput, DropdownInput, MessageInput, MultilineInput, SecretStrInput, SliderInput\nfrom langflow.io import Output\nfrom langflow.schema.message import Message\n\n\nclass TaskError(Exception):\n \"\"\"Error raised when a task fails.\"\"\"\n\n\nclass TaskTimeoutError(Exception):\n \"\"\"Error raised when a task times out.\"\"\"\n\n\nclass IndexCreationError(Exception):\n \"\"\"Error raised when there's an issue with an index.\"\"\"\n\n\nclass ApiRequestError(Exception):\n \"\"\"Error raised when an API request fails.\"\"\"\n\n\nclass VideoValidationError(Exception):\n \"\"\"Error raised when video validation fails.\"\"\"\n\n\nclass TwelveLabsPegasus(Component):\n display_name = \"Twelve Labs Pegasus\"\n description = \"Chat with videos using Twelve Labs Pegasus API.\"\n icon = \"TwelveLabs\"\n name = \"TwelveLabsPegasus\"\n documentation = \"https://github.com/twelvelabs-io/twelvelabs-developer-experience/blob/main/integrations/Langflow/TWELVE_LABS_COMPONENTS_README.md\"\n\n inputs = [\n DataInput(name=\"videodata\", display_name=\"Video Data\", info=\"Video Data\", is_list=True),\n SecretStrInput(\n name=\"api_key\", display_name=\"Twelve Labs API Key\", info=\"Enter your Twelve Labs API Key.\", required=True\n ),\n MessageInput(\n name=\"video_id\",\n display_name=\"Pegasus Video ID\",\n info=\"Enter a Video ID for a previously indexed video.\",\n ),\n MessageInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n info=\"Name of the index to use. If the index doesn't exist, it will be created.\",\n required=False,\n ),\n MessageInput(\n name=\"index_id\",\n display_name=\"Index ID\",\n info=\"ID of an existing index to use. If provided, index_name will be ignored.\",\n required=False,\n ),\n DropdownInput(\n name=\"model_name\",\n display_name=\"Model\",\n info=\"Pegasus model to use for indexing\",\n options=[\"pegasus1.2\"],\n value=\"pegasus1.2\",\n advanced=False,\n ),\n MultilineInput(\n name=\"message\",\n display_name=\"Prompt\",\n info=\"Message to chat with the video.\",\n required=True,\n ),\n SliderInput(\n name=\"temperature\",\n display_name=\"Temperature\",\n value=0.7,\n range_spec=RangeSpec(min=0, max=1, step=0.01),\n info=(\n \"Controls randomness in responses. Lower values are more deterministic, \"\n \"higher values are more creative.\"\n ),\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"Message\",\n name=\"response\",\n method=\"process_video\",\n type_=Message,\n ),\n Output(\n display_name=\"Video ID\",\n name=\"processed_video_id\",\n method=\"get_video_id\",\n type_=Message,\n ),\n ]\n\n def __init__(self, **kwargs) -> None:\n super().__init__(**kwargs)\n\n self._task_id: str | None = None\n self._video_id: str | None = None\n self._index_id: str | None = None\n self._index_name: str | None = None\n self._message: str | None = None\n\n def _get_or_create_index(self, client: TwelveLabs) -> tuple[str, str]:\n \"\"\"Get existing index or create new one.\n\n Returns (index_id, index_name).\n \"\"\"\n # First check if index_id is provided and valid\n if hasattr(self, \"_index_id\") and self._index_id:\n try:\n index = client.index.retrieve(id=self._index_id)\n self.log(f\"Found existing index with ID: {self._index_id}\")\n except (ValueError, KeyError) as e:\n self.log(f\"Error retrieving index with ID {self._index_id}: {e!s}\", \"WARNING\")\n else:\n return self._index_id, index.name\n\n # If index_name is provided, try to find it\n if hasattr(self, \"_index_name\") and self._index_name:\n try:\n # List all indexes and find by name\n indexes = client.index.list()\n for idx in indexes:\n if idx.name == self._index_name:\n self.log(f\"Found existing index: {self._index_name} (ID: {idx.id})\")\n return idx.id, idx.name\n\n # If we get here, index wasn't found - create it\n self.log(f\"Creating new index: {self._index_name}\")\n index = client.index.create(\n name=self._index_name,\n models=[\n {\n \"name\": self.model_name if hasattr(self, \"model_name\") else \"pegasus1.2\",\n \"options\": [\"visual\", \"audio\"],\n }\n ],\n )\n except (ValueError, KeyError) as e:\n self.log(f\"Error with index name {self._index_name}: {e!s}\", \"ERROR\")\n error_message = f\"Error with index name {self._index_name}\"\n raise IndexCreationError(error_message) from e\n else:\n return index.id, index.name\n\n # If neither is provided, create a new index with timestamp\n try:\n index_name = f\"index_{int(time.time())}\"\n self.log(f\"Creating new index: {index_name}\")\n index = client.index.create(\n name=index_name,\n models=[\n {\n \"name\": self.model_name if hasattr(self, \"model_name\") else \"pegasus1.2\",\n \"options\": [\"visual\", \"audio\"],\n }\n ],\n )\n except (ValueError, KeyError) as e:\n self.log(f\"Failed to create new index: {e!s}\", \"ERROR\")\n error_message = \"Failed to create new index\"\n raise IndexCreationError(error_message) from e\n else:\n return index.id, index.name\n\n @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10), reraise=True)\n async def _make_api_request(self, method: Any, *args: Any, **kwargs: Any) -> Any:\n \"\"\"Make API request with retry logic.\n\n Retries failed requests with exponential backoff.\n \"\"\"\n try:\n return await method(*args, **kwargs)\n except (ValueError, KeyError) as e:\n self.log(f\"API request failed: {e!s}\", \"ERROR\")\n error_message = \"API request failed\"\n raise ApiRequestError(error_message) from e\n\n def wait_for_task_completion(\n self, client: TwelveLabs, task_id: str, max_retries: int = 120, sleep_time: int = 5\n ) -> Any:\n \"\"\"Wait for task completion with timeout and improved error handling.\n\n Polls the task status until completion or timeout.\n \"\"\"\n retries = 0\n consecutive_errors = 0\n max_consecutive_errors = 3\n\n while retries < max_retries:\n try:\n self.log(f\"Checking task status (attempt {retries + 1})\")\n result = client.task.retrieve(id=task_id)\n consecutive_errors = 0 # Reset error counter on success\n\n if result.status == \"ready\":\n self.log(\"Task completed successfully!\")\n return result\n if result.status == \"failed\":\n error_msg = f\"Task failed with status: {result.status}\"\n self.log(error_msg, \"ERROR\")\n raise TaskError(error_msg)\n if result.status == \"error\":\n error_msg = f\"Task encountered an error: {getattr(result, 'error', 'Unknown error')}\"\n self.log(error_msg, \"ERROR\")\n raise TaskError(error_msg)\n\n time.sleep(sleep_time)\n retries += 1\n status_msg = f\"Processing video... {retries * sleep_time}s elapsed\"\n self.status = status_msg\n self.log(status_msg)\n\n except (ValueError, KeyError) as e:\n consecutive_errors += 1\n error_msg = f\"Error checking task status: {e!s}\"\n self.log(error_msg, \"WARNING\")\n\n if consecutive_errors >= max_consecutive_errors:\n too_many_errors = \"Too many consecutive errors\"\n raise TaskError(too_many_errors) from e\n\n time.sleep(sleep_time * 2)\n continue\n\n timeout_msg = f\"Timeout after {max_retries * sleep_time} seconds\"\n self.log(timeout_msg, \"ERROR\")\n raise TaskTimeoutError(timeout_msg)\n\n def validate_video_file(self, filepath: str) -> tuple[bool, str]:\n \"\"\"Validate video file using ffprobe.\n\n Returns (is_valid, error_message).\n \"\"\"\n # Ensure filepath is a string and doesn't contain shell metacharacters\n if not isinstance(filepath, str) or any(c in filepath for c in \";&|`$(){}[]<>*?!#~\"):\n return False, \"Invalid filepath\"\n\n try:\n cmd = [\n \"ffprobe\",\n \"-loglevel\",\n \"error\",\n \"-show_entries\",\n \"stream=codec_type,codec_name\",\n \"-of\",\n \"default=nw=1\",\n \"-print_format\",\n \"json\",\n \"-show_format\",\n filepath,\n ]\n\n # Use subprocess with a list of arguments to avoid shell injection\n # We need to skip the S603 warning here as we're taking proper precautions\n # with input validation and using shell=False\n result = subprocess.run( # noqa: S603\n cmd,\n capture_output=True,\n text=True,\n check=False,\n shell=False, # Explicitly set shell=False for security\n )\n\n if result.returncode != 0:\n return False, f\"FFprobe error: {result.stderr}\"\n\n probe_data = json.loads(result.stdout)\n\n has_video = any(stream.get(\"codec_type\") == \"video\" for stream in probe_data.get(\"streams\", []))\n\n if not has_video:\n return False, \"No video stream found in file\"\n\n self.log(f\"Video validation successful: {json.dumps(probe_data, indent=2)}\")\n except subprocess.SubprocessError as e:\n return False, f\"FFprobe process error: {e!s}\"\n except json.JSONDecodeError as e:\n return False, f\"FFprobe output parsing error: {e!s}\"\n except (ValueError, OSError) as e:\n return False, f\"Validation error: {e!s}\"\n else:\n return True, \"\"\n\n def on_task_update(self, task: Any) -> None:\n \"\"\"Callback for task status updates.\n\n Updates the component status with the current task status.\n \"\"\"\n self.status = f\"Processing video... Status: {task.status}\"\n self.log(self.status)\n\n def process_video(self) -> Message:\n \"\"\"Process video using Pegasus and generate response if message is provided.\n\n Handles video indexing and question answering using the Twelve Labs API.\n \"\"\"\n # Check and initialize inputs\n if hasattr(self, \"index_id\") and self.index_id:\n self._index_id = self.index_id.text if hasattr(self.index_id, \"text\") else self.index_id\n\n if hasattr(self, \"index_name\") and self.index_name:\n self._index_name = self.index_name.text if hasattr(self.index_name, \"text\") else self.index_name\n\n if hasattr(self, \"video_id\") and self.video_id:\n self._video_id = self.video_id.text if hasattr(self.video_id, \"text\") else self.video_id\n\n if hasattr(self, \"message\") and self.message:\n self._message = self.message.text if hasattr(self.message, \"text\") else self.message\n\n try:\n # If we have a message and already processed video, use existing video_id\n if self._message and self._video_id and self._video_id != \"\":\n self.status = f\"Have video id: {self._video_id}\"\n\n client = TwelveLabs(api_key=self.api_key)\n\n self.status = f\"Processing query (w/ video ID): {self._video_id} {self._message}\"\n self.log(self.status)\n\n response = client.generate.text(\n video_id=self._video_id,\n prompt=self._message,\n temperature=self.temperature,\n )\n return Message(text=response.data)\n\n # Otherwise process new video\n if not self.videodata or not isinstance(self.videodata, list) or len(self.videodata) != 1:\n return Message(text=\"Please provide exactly one video\")\n\n video_path = self.videodata[0].data.get(\"text\")\n if not video_path or not Path(video_path).exists():\n return Message(text=\"Invalid video path\")\n\n if not self.api_key:\n return Message(text=\"No API key provided\")\n\n client = TwelveLabs(api_key=self.api_key)\n\n # Get or create index\n try:\n index_id, index_name = self._get_or_create_index(client)\n self.status = f\"Using index: {index_name} (ID: {index_id})\"\n self.log(f\"Using index: {index_name} (ID: {index_id})\")\n self._index_id = index_id\n self._index_name = index_name\n except IndexCreationError as e:\n return Message(text=f\"Failed to get/create index: {e}\")\n\n with Path(video_path).open(\"rb\") as video_file:\n task = client.task.create(index_id=self._index_id, file=video_file)\n self._task_id = task.id\n\n # Wait for processing to complete\n task.wait_for_done(sleep_interval=5, callback=self.on_task_update)\n\n if task.status != \"ready\":\n return Message(text=f\"Processing failed with status {task.status}\")\n\n # Store video_id for future use\n self._video_id = task.video_id\n\n # Generate response if message provided\n if self._message:\n self.status = f\"Processing query: {self._message}\"\n self.log(self.status)\n\n response = client.generate.text(\n video_id=self._video_id,\n prompt=self._message,\n temperature=self.temperature,\n )\n return Message(text=response.data)\n\n success_msg = (\n f\"Video processed successfully. You can now ask questions about the video. Video ID: {self._video_id}\"\n )\n return Message(text=success_msg)\n\n except (ValueError, KeyError, IndexCreationError, TaskError, TaskTimeoutError) as e:\n self.log(f\"Error: {e!s}\", \"ERROR\")\n # Clear stored IDs on error\n self._video_id = None\n self._index_id = None\n self._task_id = None\n return Message(text=f\"Error: {e!s}\")\n\n def get_video_id(self) -> Message:\n \"\"\"Return the video ID of the processed video as a Message.\n\n Returns an empty string if no video has been processed.\n \"\"\"\n video_id = self._video_id or \"\"\n return Message(text=video_id)\n" }, "index_id": { "_input_type": "MessageInput", "advanced": false, "display_name": "Index ID", "dynamic": false, "info": "ID of an existing index to use. If provided, index_name will be ignored.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "index_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "index_name": { "_input_type": "MessageInput", "advanced": false, "display_name": "Index Name", "dynamic": false, "info": "Name of the index to use. If the index doesn't exist, it will be created.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "index_name", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "message": { "_input_type": "MultilineInput", "advanced": false, "copy_field": false, "display_name": "Prompt", "dynamic": false, "info": "Message to chat with the video.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "multiline": true, "name": "message", "placeholder": "", "required": true, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "" }, "model_name": { "_input_type": "DropdownInput", "advanced": false, "combobox": false, "dialog_inputs": {}, "display_name": "Model", "dynamic": false, "info": "Pegasus model to use for indexing", "name": "model_name", "options": [ "pegasus1.2" ], "options_metadata": [], "placeholder": "", "required": false, "show": true, "title_case": false, "toggle": false, "tool_mode": false, "trace_as_metadata": true, "type": "str", "value": "pegasus1.2" }, "temperature": { "_input_type": "SliderInput", "advanced": false, "display_name": "Temperature", "dynamic": false, "info": "Controls randomness in responses. Lower values are more deterministic, higher values are more creative.", "max_label": "", "max_label_icon": "", "min_label": "", "min_label_icon": "", "name": "temperature", "placeholder": "", "range_spec": { "max": 1, "min": 0, "step": 0.01, "step_type": "float" }, "required": false, "show": true, "slider_buttons": false, "slider_buttons_options": [], "slider_input": false, "title_case": false, "tool_mode": false, "type": "slider", "value": 0.33 }, "video_id": { "_input_type": "MessageInput", "advanced": false, "display_name": "Pegasus Video ID", "dynamic": false, "info": "Enter a Video ID for a previously indexed video.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", "load_from_db": false, "name": "video_id", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "str", "value": "683b3fc704cb7d91a04da65e" }, "videodata": { "_input_type": "DataInput", "advanced": false, "display_name": "Video Data", "dynamic": false, "info": "Video Data", "input_types": [ "Data" ], "list": true, "list_add_label": "Add More", "name": "videodata", "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "type": "other", "value": "" } }, "tool_mode": false }, "showNode": true, "type": "TwelveLabsPegasus" }, "dragging": false, "id": "TwelveLabsPegasus-8YZUt", "measured": { "height": 843, "width": 320 }, "position": { "x": 1812.8998103788715, "y": -121.92241734096879 }, "selected": false, "type": "genericNode" } ], "viewport": { "x": -70.73752956727867, "y": 160.53322836788237, "zoom": 0.39558836655237256 } }, "description": "Conversational Cartography Unlocked.", "endpoint_name": null, "id": "20e482e7-b276-4290-a635-55d6627a685a", "is_component": false, "last_tested_version": "1.4.2", "name": "Stitch Flow", "tags": [] }